Packages and utilities

library(tidyverse)
Registered S3 method overwritten by 'dplyr':
  method           from
  print.rowwise_df     
Registered S3 methods overwritten by 'dbplyr':
  method         from
  print.tbl_lazy     
  print.tbl_sql      
── Attaching packages ────────────────────────────────────────────────────────────────────── tidyverse 1.3.0 ──
✓ ggplot2 3.2.1     ✓ purrr   0.3.3
✓ tibble  2.1.3     ✓ dplyr   0.8.4
✓ tidyr   1.0.2     ✓ stringr 1.4.0
✓ readr   1.3.1     ✓ forcats 0.4.0
── Conflicts ───────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
x dplyr::filter() masks stats::filter()
x dplyr::lag()    masks stats::lag()
library(lme4)
Loading required package: Matrix

Attaching package: ‘Matrix’

The following objects are masked from ‘package:tidyr’:

    expand, pack, unpack
library(lmerTest)

Attaching package: ‘lmerTest’

The following object is masked from ‘package:lme4’:

    lmer

The following object is masked from ‘package:stats’:

    step
library(plotrix)
library(stringr)
library(readxl)
library(RColorBrewer)
library(mvtnorm)
library(mgcv)
Loading required package: nlme

Attaching package: ‘nlme’

The following object is masked from ‘package:lme4’:

    lmList

The following object is masked from ‘package:dplyr’:

    collapse

This is mgcv 1.8-31. For overview type 'help("mgcv-package")'.
# Compute the log-likelihood of a new dataset using a fit lme4 model.
logLik_test <- function(lm, test_X, test_y) {
  predictions <- predict(lm, test_X, re.form=NA)
  # Get std.dev. of residual, estimated from train data
  stdev <- sigma(lm)
  # For each prediction--observation, get the density p(obs | N(predicted, model_sigma)) and reduce
  density <- sum(dnorm(test_y, predictions, stdev, log=TRUE))
  return(density)
}
# Get per-prediction log-likelihood
logLik_test_per <- function(lm, test_X, test_y) {
  predictions <- predict(lm, test_X, re.form=NA)
  # Get std.dev. of residual, estimated from train data
  stdev <- sigma(lm)
  # For each prediction--observation, get the density p(obs | N(predicted, model_sigma))
  densities <- dnorm(test_y, predictions, stdev, log=TRUE)
  return(densities)
}
# Compute MSE of a new dataset using a fit lme4 model.
mse_test <- function(lm, test_X, test_y) {
  return(mean((predict(lm, test_X, re.form=NA) - test_y) ^ 2))
}
#Sanity checks
#mylm <- gam(psychometric ~  s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr"), data=train_data)
#c(logLik(mylm), logLik_test(mylm, train_data, train_data$psychometric))
#logLik_test(mylm, test_data, test_data$psychometric)

Data loading and preprocessing

data = read.csv("../data/harmonized_results.csv")

all_data = data %>%
  mutate(seed = as.factor(seed)) %>%
  group_by(corpus, model, training, seed) %>%
    mutate(prev_surp = lag(surprisal),
         prev_code = lag(code),
         prev_len = lag(len),
         prev_freq = lag(freq),
         prev_surp = lag(surprisal),
         
         prev2_freq = lag(prev_freq),
         prev2_code = lag(prev_code),
         prev2_len = lag(prev_len),
         prev2_surp = lag(prev_surp),
         
         prev3_freq = lag(prev2_freq),
         prev3_code = lag(prev2_code),
         prev3_len = lag(prev2_len),
         prev3_surp = lag(prev2_surp)) %>%
  ungroup() %>%
  # Filter back three for the dundee corpus. Filter back 1 for all other corpora
  filter((corpus == "dundee" & code == prev2_code + 2) | (corpus != "dundee" & code == prev_code + 1)) %>%
  select(-prev_code, -prev2_code, -prev3_code) %>%
  drop_na()

all_data = all_data %>%
  mutate(
    model = as.character(model),
    model = if_else(model == "gpt-2", "gpt2", model),
    model = as.factor(model))
  
# Compute linear model stats for the given training data subset and full test data.
# Automatically subsets the test data to match the relevant group for which we are training a linear model.
get_lm_data <- function(df, test_data, formula, store_env) {
  #this_lm <- gam(formula, data=df);
  this_lm = lm(formula, data=df)
  this_test_data <- semi_join(test_data, df, by=c("training", "model", "seed", "corpus"));
  
  # Save lm to the global env so that we can access residuals later.
  lm_name = unique(paste(df$model, df$training, df$seed, df$corpus))[1]
  assign(lm_name, this_lm, envir=store_env)
  
  summarise(df,
            log_lik = as.numeric(logLik(this_lm, REML = F)),
            test_lik = logLik_test(this_lm, this_test_data, this_test_data$psychometric),
            test_mse = mse_test(this_lm, this_test_data, this_test_data$psychometric))
}
# For a previously fitted lm stored in store_env, get the residuals on test data of the relevant data subset.
get_lm_residuals <- function(df, store_env) {
  # Retrieve the relevant lm.
  lm_name = unique(paste(df$model, df$training, df$seed, df$corpus))[1]
  this_lm <- get(lm_name, envir=store_env)
  
  mutate(df,
         likelihood = logLik_test_per(this_lm, df, df$psychometric),
         resid = df$psychometric - predict(this_lm, df, re.form=NA))
}
#####
# Define regression formulae.
# Eye-tracking regression: only use surprisal and previous surprisal; SPRT regression: use 2-back features.
#baseline_rt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr")
#baselie_sprt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr")
#full_rt_regression = (psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20)
                     #+ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr"))
#full_sprt_regression = (psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + s(prev2_surp, bs = "cr", k = 20)
                        #+ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr"))

baseline_rt_regression = psychometric ~ freq + prev_freq + prev2_freq + prev3_freq + len + prev_len + prev2_len + prev3_len
baseline_sprt_regression = psychometric ~ freq + prev_freq + len + prev_len

full_rt_regression = psychometric ~ surprisal + prev_surp + prev2_surp + prev3_surp + freq + prev_freq + prev2_freq + prev3_freq + len + prev_len + prev2_len + prev3_len
full_sprt_regression = psychometric ~ surprisal + prev_surp + freq + prev_freq + len + prev_len
  
#####
# Prepare frames/environments for storing results/objects.
baseline_results = data.frame()
full_model_results = data.frame()
baseline_residuals = data.frame()
full_residuals = data.frame()
#Randomly shuffle the data
all_data<-all_data[sample(nrow(all_data)),]
#Create K equally size folds
K = 5
folds <- cut(seq(1,nrow(all_data)),breaks=K,labels=FALSE)
#Perform 10 fold cross validation
baseline_corpus = function(corpus, df, test_data, env) {
  if(corpus == "dundee") {
    get_lm_data(df, test_data, baseline_rt_regression, env)
  } else {
    get_lm_data(df, test_data, baseline_sprt_regression, env)
  }
}
full_model_corpus = function(corpus, df, test_data, env) {
  if(corpus[1] == "dundee") {
    get_lm_data(df, test_data, full_rt_regression, env)
  } else {
    get_lm_data(df, test_data, full_sprt_regression, env)
  }
}
for(i in 1:K) { 
  #Segement your data by fold using the which() function 
  testIndexes <- which(folds==i,arr.ind=TRUE)
  test_data <- all_data[testIndexes, ]
  train_data <- all_data[-testIndexes, ]
  
  # Prepare a new Environment in which we store fitted LMs, which we'll query later for residuals.
  baseline_env = environment()
  full_env = environment()
  
  # Compute a baseline linear model for each model--training--seed--RT-corpus combination.
  baselines = train_data %>%
    group_by(model, training, seed, corpus) %>%
      print(model) %>%
      do(baseline_corpus(unique(.$corpus), ., test_data, baseline_env)) %>%
    ungroup() %>%
    mutate(seed = as.factor(seed),
           fold = i)
  
  baseline_results = rbind(baseline_results, baselines)
  
  # Compute a full linear model for each model--training--seed-RT-corpus combination
  full_models = train_data %>%
    group_by(model, training, seed, corpus) %>%
      do(full_model_corpus(unique(.$corpus), ., test_data, full_env)) %>%
    ungroup() %>%
    mutate(seed = as.factor(seed),
           fold = i)
  
  full_model_results = rbind(full_model_results, full_models)
  
  fold_baseline_residuals = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_residuals(., baseline_env)) %>%
    ungroup()
  
  baseline_residuals = rbind(baseline_residuals, fold_baseline_residuals)
  
  fold_full_residuals = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_residuals(., full_env)) %>%
    ungroup() 
  
  full_residuals = rbind(full_residuals, fold_full_residuals)
}

|=========================================================================================                                                                                       | 51% ~2 s remaining     
|============================================================================================                                                                                    | 52% ~2 s remaining     
|=================================================================================================                                                                               | 55% ~2 s remaining     
|======================================================================================================                                                                          | 58% ~2 s remaining     
|============================================================================================================                                                                    | 62% ~2 s remaining     
|=================================================================================================================                                                               | 65% ~1 s remaining     
|=======================================================================================================================                                                         | 68% ~1 s remaining     
|============================================================================================================================                                                    | 71% ~1 s remaining     
|=================================================================================================================================                                               | 74% ~1 s remaining     
|=======================================================================================================================================                                         | 77% ~1 s remaining     
|============================================================================================================================================                                    | 80% ~1 s remaining     
|==================================================================================================================================================                              | 83% ~1 s remaining     
|=======================================================================================================================================================                         | 86% ~1 s remaining     
|=============================================================================================================================================================                   | 89% ~0 s remaining     
|==================================================================================================================================================================              | 92% ~0 s remaining     
|=======================================================================================================================================================================         | 95% ~0 s remaining     
|=============================================================================================================================================================================   | 98% ~0 s remaining     
|==============================================================================================                                                                                  | 54% ~2 s remaining     
|=================================================================================================                                                                               | 55% ~2 s remaining     
|======================================================================================================                                                                          | 58% ~2 s remaining     
|============================================================================================================                                                                    | 62% ~1 s remaining     
|=================================================================================================================                                                               | 65% ~1 s remaining     
|=======================================================================================================================                                                         | 68% ~1 s remaining     
|============================================================================================================================                                                    | 71% ~1 s remaining     
|=================================================================================================================================                                               | 74% ~1 s remaining     
|=======================================================================================================================================                                         | 77% ~1 s remaining     
|============================================================================================================================================                                    | 80% ~1 s remaining     
|==================================================================================================================================================                              | 83% ~1 s remaining     
|=======================================================================================================================================================                         | 86% ~1 s remaining     
|=============================================================================================================================================================                   | 89% ~0 s remaining     
|==================================================================================================================================================================              | 92% ~0 s remaining     
|=======================================================================================================================================================================         | 95% ~0 s remaining     
|=============================================================================================================================================================================   | 98% ~0 s remaining     

|======================================================================================================                                                                          | 58% ~1 s remaining     
|============================================================================================================                                                                    | 62% ~1 s remaining     
|=================================================================================================================                                                               | 65% ~1 s remaining     
|=======================================================================================================================                                                         | 68% ~1 s remaining     
|============================================================================================================================                                                    | 71% ~1 s remaining     
|=================================================================================================================================                                               | 74% ~1 s remaining     
|=======================================================================================================================================                                         | 77% ~1 s remaining     
|============================================================================================================================================                                    | 80% ~1 s remaining     
|==================================================================================================================================================                              | 83% ~1 s remaining     
|=======================================================================================================================================================                         | 86% ~0 s remaining     
|=============================================================================================================================================================                   | 89% ~0 s remaining     
|==================================================================================================================================================================              | 92% ~0 s remaining     
|=====================================================================================================================================================================           | 94% ~0 s remaining     
|=======================================================================================================================================================================         | 95% ~0 s remaining     
|=============================================================================================================================================================================   | 98% ~0 s remaining     
|==============================================================================                                                                                                  | 45% ~3 s remaining     
|=================================================================================                                                                                               | 46% ~2 s remaining     
|======================================================================================                                                                                          | 49% ~2 s remaining     
|============================================================================================                                                                                    | 52% ~2 s remaining     
|=================================================================================================                                                                               | 55% ~2 s remaining     
|======================================================================================================                                                                          | 58% ~2 s remaining     
|============================================================================================================                                                                    | 62% ~2 s remaining     
|=================================================================================================================                                                               | 65% ~2 s remaining     
|=======================================================================================================================                                                         | 68% ~1 s remaining     
|============================================================================================================================                                                    | 71% ~1 s remaining     
|=================================================================================================================================                                               | 74% ~1 s remaining     
|=======================================================================================================================================                                         | 77% ~1 s remaining     
|============================================================================================================================================                                    | 80% ~1 s remaining     
|==================================================================================================================================================                              | 83% ~1 s remaining     
|=======================================================================================================================================================                         | 86% ~1 s remaining     
|=============================================================================================================================================================                   | 89% ~0 s remaining     
|==================================================================================================================================================================              | 92% ~0 s remaining     
|=======================================================================================================================================================================         | 95% ~0 s remaining     
|=============================================================================================================================================================================   | 98% ~0 s remaining     

|======================================================================================================                                                                          | 58% ~1 s remaining     
|============================================================================================================                                                                    | 62% ~1 s remaining     
|=================================================================================================================                                                               | 65% ~1 s remaining     
|=======================================================================================================================                                                         | 68% ~1 s remaining     
|============================================================================================================================                                                    | 71% ~1 s remaining     
|=================================================================================================================================                                               | 74% ~1 s remaining     
|=======================================================================================================================================                                         | 77% ~1 s remaining     
|============================================================================================================================================                                    | 80% ~1 s remaining     
|==================================================================================================================================================                              | 83% ~1 s remaining     
|=======================================================================================================================================================                         | 86% ~0 s remaining     
|=============================================================================================================================================================                   | 89% ~0 s remaining     
|==================================================================================================================================================================              | 92% ~0 s remaining     
|=======================================================================================================================================================================         | 95% ~0 s remaining     
|=============================================================================================================================================================================   | 98% ~0 s remaining     
|==============================================================================================                                                                                  | 54% ~2 s remaining     
|=================================================================================================                                                                               | 55% ~2 s remaining     
|======================================================================================================                                                                          | 58% ~2 s remaining     
|============================================================================================================                                                                    | 62% ~1 s remaining     
|=================================================================================================================                                                               | 65% ~1 s remaining     
|=======================================================================================================================                                                         | 68% ~1 s remaining     
|============================================================================================================================                                                    | 71% ~1 s remaining     
|=================================================================================================================================                                               | 74% ~1 s remaining     
|=======================================================================================================================================                                         | 77% ~1 s remaining     
|============================================================================================================================================                                    | 80% ~1 s remaining     
|==================================================================================================================================================                              | 83% ~1 s remaining     
|====================================================================================================================================================                            | 85% ~1 s remaining     
|=======================================================================================================================================================                         | 86% ~1 s remaining     
|=============================================================================================================================================================                   | 89% ~0 s remaining     
|==================================================================================================================================================================              | 92% ~0 s remaining     
|=======================================================================================================================================================================         | 95% ~0 s remaining     
|=============================================================================================================================================================================   | 98% ~0 s remaining     

|======================================================================================================                                                                          | 58% ~1 s remaining     
|============================================================================================================                                                                    | 62% ~1 s remaining     
|=================================================================================================================                                                               | 65% ~1 s remaining     
|====================================================================================================================                                                            | 66% ~1 s remaining     
|=======================================================================================================================                                                         | 68% ~1 s remaining     
|============================================================================================================================                                                    | 71% ~1 s remaining     
|=================================================================================================================================                                               | 74% ~1 s remaining     
|=======================================================================================================================================                                         | 77% ~1 s remaining     
|============================================================================================================================================                                    | 80% ~1 s remaining     
|==================================================================================================================================================                              | 83% ~1 s remaining     
|=======================================================================================================================================================                         | 86% ~0 s remaining     
|=============================================================================================================================================================                   | 89% ~0 s remaining     
|==================================================================================================================================================================              | 92% ~0 s remaining     
|=====================================================================================================================================================================           | 94% ~0 s remaining     
|=======================================================================================================================================================================         | 95% ~0 s remaining     
|=============================================================================================================================================================================   | 98% ~0 s remaining     
|=================================================================================================                                                                               | 55% ~2 s remaining     
|======================================================================================================                                                                          | 58% ~2 s remaining     
|============================================================================================================                                                                    | 62% ~1 s remaining     
|=================================================================================================================                                                               | 65% ~1 s remaining     
|=======================================================================================================================                                                         | 68% ~1 s remaining     
|============================================================================================================================                                                    | 71% ~1 s remaining     
|=================================================================================================================================                                               | 74% ~1 s remaining     
|=======================================================================================================================================                                         | 77% ~1 s remaining     
|============================================================================================================================================                                    | 80% ~1 s remaining     
|===============================================================================================================================================                                 | 82% ~1 s remaining     
|==================================================================================================================================================                              | 83% ~1 s remaining     
|=======================================================================================================================================================                         | 86% ~1 s remaining     
|=============================================================================================================================================================                   | 89% ~0 s remaining     
|==================================================================================================================================================================              | 92% ~0 s remaining     
|=======================================================================================================================================================================         | 95% ~0 s remaining     
|=============================================================================================================================================================================   | 98% ~0 s remaining     

|======================================================================================================                                                                          | 58% ~1 s remaining     
|============================================================================================================                                                                    | 62% ~1 s remaining     
|=================================================================================================================                                                               | 65% ~1 s remaining     
|=======================================================================================================================                                                         | 68% ~1 s remaining     
|============================================================================================================================                                                    | 71% ~1 s remaining     
|=================================================================================================================================                                               | 74% ~1 s remaining     
|=======================================================================================================================================                                         | 77% ~1 s remaining     
|============================================================================================================================================                                    | 80% ~1 s remaining     
|==================================================================================================================================================                              | 83% ~1 s remaining     
|=======================================================================================================================================================                         | 86% ~0 s remaining     
|=============================================================================================================================================================                   | 89% ~0 s remaining     
|==================================================================================================================================================================              | 92% ~0 s remaining     
|=======================================================================================================================================================================         | 95% ~0 s remaining     
|=============================================================================================================================================================================   | 98% ~0 s remaining     
|============================================================================================                                                                                    | 52% ~2 s remaining     
|=================================================================================================                                                                               | 55% ~2 s remaining     
|======================================================================================================                                                                          | 58% ~2 s remaining     
|============================================================================================================                                                                    | 62% ~1 s remaining     
|=================================================================================================================                                                               | 65% ~1 s remaining     
|=======================================================================================================================                                                         | 68% ~1 s remaining     
|============================================================================================================================                                                    | 71% ~1 s remaining     
|===============================================================================================================================                                                 | 72% ~1 s remaining     
|=================================================================================================================================                                               | 74% ~1 s remaining     
|=======================================================================================================================================                                         | 77% ~1 s remaining     
|============================================================================================================================================                                    | 80% ~1 s remaining     
|==================================================================================================================================================                              | 83% ~1 s remaining     
|=======================================================================================================================================================                         | 86% ~1 s remaining     
|=============================================================================================================================================================                   | 89% ~0 s remaining     
|==================================================================================================================================================================              | 92% ~0 s remaining     
|=======================================================================================================================================================================         | 95% ~0 s remaining     
|=============================================================================================================================================================================   | 98% ~0 s remaining     
write.csv(full_residuals, "../data/analysis_checkpoints/full_residuals.csv")
write.csv(baseline_residuals, "../data/analysis_checkpoints/baseline_residuals.csv")
write.csv(full_model_results, "../data/analysis_checkpoints/full_model_result.csv")
write.csv(baseline_results, "../data/analysis_checkpoints/baseline_results.csv")
#full_model_results = read.csv("../data/analysis_checkpoints/ffull_model_results.csv")
#baseline_results = read.csv("../data/analysis_checkpoints/fbaseline_resultsb.csv")

# Join baseline models with full models and compare performance within-fold.
model_fold_deltas = baseline_results %>%
  right_join(full_model_results, suffix=c(".baseline", ".full"),
             by=c("model", "training", "seed", "corpus", "fold")) %>%
  
  mutate(seed = as.factor(seed)) %>%
  
  # Compute per-fold deltas.
  group_by(model, training, seed, corpus, fold) %>%
    mutate(delta_log_lik = test_lik.full - test_lik.baseline,
           delta_mse = test_mse.full - test_mse.baseline) %>%
  ungroup() %>%
  select(model, training, seed, corpus, fold,
         delta_log_lik, delta_mse)

# Now compute across-fold delta statistics for each model--training--seed--corpus.
model_deltas = model_fold_deltas %>%
  group_by(model, training, seed, corpus) %>%
    summarise(mean_delta_log_lik = sum(delta_log_lik),
              sem_delta_log_lik = sd(delta_log_lik) / sqrt(length(delta_log_lik)),
              mean_delta_mse = sum(delta_mse),
              sem_delta_mse = sd(delta_mse) / sqrt(length(delta_mse)))
metric <- "ΔLogLik"
#metric <- "-ΔMSE"
# Select the relevant metric.
model_fold_deltas = model_fold_deltas %>%
  # Retrieve the current test metric
  mutate(delta_test = delta_log_lik) %>%
  select(-delta_log_lik, -delta_mse)
# Select the relevant metric.
model_deltas = model_deltas %>%
    # Retrieve the current test metric
    mutate(delta_test_mean = mean_delta_log_lik,
           delta_test_sem = sem_delta_log_lik) %>%
    # mutate(delta_test_mean = mean_delta_mse,
    #        delta_test_sem = sem_delta_mse)
    
    # Remove the raw metrics.
    select(-mean_delta_log_lik, -sem_delta_log_lik,
           -mean_delta_mse, -sem_delta_mse)
model_deltas
# Sanity check: training on train+test data should yield improved performance over training on just training data. (When evaluating on test data.)
# full_baselines = all_data %>%
#   group_by(model, training, seed, corpus) %>%
#   summarise(baseline_train_all_test_lik = logLik_test(lm(psychometric ~ len + freq + sent_pos, data=.), semi_join(test_data, ., by=c("training", "model", "seed", "corpus")), semi_join(test_data, ., by=c("training", "model", "seed", "corpus"))$psychometric)) %>%
#   ungroup()
# full_baselines
# 
# full_baselines %>%
#   right_join(baselines, by=c("seed", "training", "model", "corpus")) %>%
#   mutate(delta=baseline_train_all_test_lik-baseline_test_lik) %>%
#   select(-baseline_lik) # %>%
#   #select(-baseline_test_lik, -baseline_train_all_test_lik, -baseline_lik, -baseline_test_mse)

Load language model data (SyntaxGym, PPL)

language_model_data = read.csv("../data/model_metadata.csv") %>%
  mutate(model = as.character(model),
         model = if_else(model == "gpt-2", "gpt2", model),
         model = as.factor(model)) %>%
  mutate(train_size = case_when(str_starts(training, "bllip-lg") ~ 42,
                                str_starts(training, "bllip-md") ~ 15,
                                str_starts(training, "bllip-sm") ~ 5,
                                str_starts(training, "bllip-xs") ~ 1)) %>%
  mutate(seed = as.factor(seed)) %>%
  select(-pid, -test_loss) %>%
  distinct(model, training, seed, .keep_all = TRUE)
table(language_model_data$seed)

         0        111        120        922       1111       3602       4301       7245       7877      28066      28068      44862      51272      64924 1581807512 1581807578 1581861474 1581955288 
         4          7          6          5          4          1          1          1          1          1          1          1          1          1          1          1          1          1 
1582126320 1586986276 1587139950 
         1          1          1 
table(model_deltas$seed)

       111        120        922       1111       3602       4301       7245       7877      28066      28068      44862      51272      64924 1581807512 1581807578 1581861474 1581955288 1582126320 
         6          6          6         10          2          2          2          2          2          2          2          2          2          3          3          3          3          3 
1586986276 1587139950 
         2          2 

First join delta-metric data with model auxiliary data.

model_deltas = model_deltas %>%
  merge(language_model_data, by = c("seed", "training", "model"), all=T) %>%
  drop_na()

model_fold_deltas = model_fold_deltas %>%
  merge(language_model_data, by = c("seed", "training", "model"), all=T) %>%
  drop_na()

model_deltas

Also join on the original linear model data, rather than collapsing to delta-metrics. This will support regressions later on that don’t collapse across folds.

Final data preprocessing

# Exclude ordered-neurons from all analyses.
model_deltas <- model_deltas %>%
  filter(model != "ordered-neurons")
model_fold_deltas <- model_fold_deltas %>%
  filter(model != "ordered-neurons")

Visualizations

Predictive power and SG

model_deltas %>%
  ggplot(aes(x=sg_score, y=delta_test_mean)) +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem)) +
    geom_smooth(method="lm", se=T) +
    geom_point(stat="identity", position="dodge", alpha=1, size=3, aes(color=training, shape=model)) +
    ylab(metric) +
    xlab("Syntax Generalization Score") +
    ggtitle("Syntactic Generalization vs. Predictive Power") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                              "bllip-md"="#39568CFF",
                              "bllip-sm"="#1F968BFF",
                              "bllip-xs"="#73D055FF",
                              "bllip-lg-gptbpe"="#888888",
                              "bllip-md-gptbpe"="#AAAAAA",
                              "bllip-sm-gptbpe"="#CCCCCC",
                              "bllip-xs-gptbpe"="#CCCCCC")) +
    facet_grid(~corpus, scales="free") +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "bottom")

#ggsave("./cogsci_images/sg_loglik.png",height=5,width=6)

Regression analyses

We control for effects of perplexity by relating the residuals of a performance ~ PPL regression to SG score.

# Prepare a residualized regression for x1 onto y, controlling for the effects of x2.
d_resid = model_fold_deltas %>%
  drop_na() %>%
  
  # Residualize delta metric w.r.t PPL for each model--training--seed--fold
  group_by(corpus) %>%
    mutate(resid.delta = resid(lm(delta_test ~ training:test_ppl))) %>%
  ungroup() %>%
  
  # Residualize SG score w.r.t. PPL for each training group
  group_by(training) %>%
    # NB no need for training:ppl interaction, since we're within-group.
    mutate(resid.sg = resid(lm(sg_score ~ test_ppl))) %>%
  ungroup() %>%
  # Compute summary statistics across model--training--seed--corpus.
  group_by(model, training, corpus, seed) %>%
    summarise(resid.delta.mean = mean(resid.delta),
              resid.delta.sem = sd(resid.delta) / sqrt(length(resid.delta)),
              resid.sg.mean = mean(resid.sg),
              resid.sg.sem = sd(resid.sg) / sqrt(length(resid.sg)))
# Now plot residual vs SG
d_resid %>%
  #filter(corpus != "bnc-brown") %>%
  ggplot(aes(x=resid.sg.mean, y=resid.delta.mean)) +
    geom_errorbar(aes(xmin=resid.sg.mean - resid.sg.sem,
                      xmax=resid.sg.mean + resid.sg.sem,
                      ymin=resid.delta.mean - resid.delta.sem,
                      ymax=resid.delta.mean + resid.delta.sem), alpha=0.3) +
    geom_smooth(method="lm", se=T) +
    geom_point(stat="identity", position="dodge", alpha=1, size=4, aes(shape=model, color=training)) +
    ylab(paste("Residual", metric)) +
    xlab("Residual Syntax Generalization Score") +
    ggtitle("Syntactic Generalization vs. Predictive Power") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "bllip-lg-gptbpe"="#888888",
                                  "bllip-md-gptbpe"="#AAAAAA",
                                  "bllip-sm-gptbpe"="#CCCCCC",
                                  "bllip-xs-gptbpe"="#CCCCCC")) +
    facet_grid(.~corpus, scales="free") +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "right")
Ignoring unknown aesthetics: xmin, xmax
ggsave("../images/cuny2020/ppl_sg.png",height=4.5,width=11)

do_stepwise_regression = function(cur_corpus) {
  regression_data = model_fold_deltas %>%
    filter(corpus == cur_corpus)
  
  # NB we're incorporating variance across folds into this regression, good!
  print("----------------------")
  print(cur_corpus)
  
  lm1 = lm(delta_test ~ training:test_ppl, data = regression_data)
  lm2 = lm(delta_test ~ training:test_ppl + sg_score, data = regression_data)
  print(anova(lm1, lm2))
  summary(lm2)
}
#do_stepwise_regression("bnc-brown")
do_stepwise_regression("dundee")
[1] "----------------------"
[1] "dundee"
Analysis of Variance Table

Model 1: delta_test ~ training:test_ppl
Model 2: delta_test ~ training:test_ppl + sg_score
  Res.Df   RSS Df Sum of Sq     F   Pr(>F)   
1    136 25890                               
2    135 24555  1    1334.2 7.335 0.007638 **
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Call:
lm(formula = delta_test ~ training:test_ppl + sg_score, data = regression_data)

Residuals:
    Min      1Q  Median      3Q     Max 
-55.316  -4.612  -0.492   4.296  48.896 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)    
(Intercept)                       30.46868    7.63911   3.989 0.000108 ***
sg_score                         -32.47411   11.99048  -2.708 0.007638 ** 
trainingbllip-lg:test_ppl          0.22829    0.06897   3.310 0.001197 ** 
trainingbllip-lg-gptbpe:test_ppl   6.73457    0.23468  28.696  < 2e-16 ***
trainingbllip-md:test_ppl          0.12484    0.05902   2.115 0.036261 *  
trainingbllip-md-gptbpe:test_ppl   5.72687    0.16970  33.747  < 2e-16 ***
trainingbllip-sm:test_ppl          0.01160    0.05160   0.225 0.822483    
trainingbllip-sm-gptbpe:test_ppl   1.33549    0.04429  30.152  < 2e-16 ***
trainingbllip-xs:test_ppl         -0.03110    0.03482  -0.893 0.373345    
trainingbllip-xs-gptbpe:test_ppl   0.44107    0.01503  29.349  < 2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 13.49 on 135 degrees of freedom
Multiple R-squared:  0.9754,    Adjusted R-squared:  0.9738 
F-statistic: 595.6 on 9 and 135 DF,  p-value: < 2.2e-16
do_stepwise_regression("natural-stories")
[1] "----------------------"
[1] "natural-stories"
Analysis of Variance Table

Model 1: delta_test ~ training:test_ppl
Model 2: delta_test ~ training:test_ppl + sg_score
  Res.Df    RSS Df Sum of Sq      F Pr(>F)
1    136 2032.6                           
2    135 2019.8  1    12.791 0.8549 0.3568

Call:
lm(formula = delta_test ~ training:test_ppl + sg_score, data = regression_data)

Residuals:
     Min       1Q   Median       3Q      Max 
-12.0616  -2.4161   0.2438   2.6515  10.6305 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)    
(Intercept)                      12.345412   2.190932   5.635 9.81e-08 ***
sg_score                         -3.179657   3.438928  -0.925 0.356820    
trainingbllip-lg:test_ppl        -0.002126   0.019782  -0.107 0.914588    
trainingbllip-lg-gptbpe:test_ppl -0.252006   0.067308  -3.744 0.000267 ***
trainingbllip-md:test_ppl        -0.024897   0.016928  -1.471 0.143683    
trainingbllip-md-gptbpe:test_ppl -0.239400   0.048671  -4.919 2.49e-06 ***
trainingbllip-sm:test_ppl        -0.047759   0.014798  -3.227 0.001568 ** 
trainingbllip-sm-gptbpe:test_ppl -0.061851   0.012703  -4.869 3.09e-06 ***
trainingbllip-xs:test_ppl        -0.044518   0.009985  -4.458 1.72e-05 ***
trainingbllip-xs-gptbpe:test_ppl -0.020918   0.004310  -4.853 3.31e-06 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 3.868 on 135 degrees of freedom
Multiple R-squared:  0.445, Adjusted R-squared:  0.4081 
F-statistic: 12.03 on 9 and 135 DF,  p-value: 8.343e-14

Predictive power and perplexity

model_deltas %>%
  ggplot(aes(x=test_ppl, y=delta_test_mean, color=training)) +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem), alpha=0.4) +
    #geom_smooth(method="lm", se=F) +
    geom_point(stat="identity", position="dodge", alpha=1, size=4, aes(shape=model)) +
    ylab(metric) +
    xlab("Test Perplexity") +
    #coord_cartesian(ylim = c(1, 16)) +
    ggtitle("Test Perplexity vs. Predictive Power") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "bllip-lg-gptbpe"="#888888",
                                  "bllip-md-gptbpe"="#AAAAAA",
                                  "bllip-sm-gptbpe"="#CCCCCC",
                                  "bllip-xs-gptbpe"="#CCCCCC")) +
    facet_grid(~corpus, scales="free") +
    #coord_cartesian(ylim = c(0, 150)) +
    theme(axis.text=element_text(size=12),
          strip.text.x = element_text(size=12),
          legend.text=element_text(size=12),
          axis.title=element_text(size=12),
          legend.position = "right")
ggsave("../images/cuny2020/ppl_loglik.png",height=4.5,width=11)

model_deltas %>%
  #filter(model != "5gram", training != "bllip-lg") %>%
  group_by(model, corpus) %>%
    #summarise(n = n())
    summarise(corr = as.numeric(cor.test(delta_test_mean, test_ppl)[4]),
              pval = as.numeric(cor.test(delta_test_mean, test_ppl)[3]))

Effect of training data size

model_deltas %>%
  mutate(train_size = log(train_size)) %>%
  ggplot(aes(x=train_size, y=delta_test_mean, color=model)) +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem), width = 0.1) +
    geom_smooth(method="lm", se=T, alpha=0.5) +
    geom_point(stat="identity", position="dodge", alpha=1, size=3) +
    ylab(metric) +
    xlab("Log Million Training Tokens") +
    ggtitle("Training Size vs. Predictive Power") +
    facet_grid(corpus~model, scales="free") +
    #scale_color_manual(values = c("#A42EF1", "#3894C8")) +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "bottom")

#ggsave("./cogsci_images/training_loglik.png",height=5,width=6)
model_deltas %>%
  mutate(train_size = log(train_size)) %>%
  ggplot(aes(x=train_size, y=sg_score, color=model)) +
    geom_smooth(method="lm", se=T, alpha=0.5) +
    geom_point(stat="identity", position="dodge", alpha=1, size=3) +
    ylab("SG SCore") +
    xlab("Log Million Training Tokens") +
    ggtitle("Training Size vs. Syntactic Generalization") +
    #scale_color_manual(values = c("#A42EF1", "#3894C8")) +
    facet_grid(~model, scales="free") +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "bottom")

#ggsave("./cogsci_images/training_sg.png",height=5,width=6)

Smith & Levy reproduction

  
all_data %>%
  filter(surprisal < 15, surprisal > 0) %>%
  ggplot(aes(x=surprisal, y=psychometric, color=training)) +
    stat_smooth(se=T, alpha=0.5) +
    #geom_errorbar(color="black", width=.2, position=position_dodge(width=.9), alpha=0.3) +
    #geom_point(stat="identity", position="dodge", alpha=1, size=3) +
    ylab("Processing Time (ms)") +
    xlab("Surprisal (bits)") +
    ggtitle("Surprisal vs. Reading Time / Gaze Duration") +
    facet_grid(corpus~model, scales = "free") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                              "bllip-md"="#39568CFF",
                              "bllip-sm"="#1F968BFF",
                              "bllip-xs"="#73D055FF",
                              "bllip-lg-gptbpe"="#888888",
                              "bllip-md-gptbpe"="#AAAAAA",
                              "bllip-sm-gptbpe"="#CCCCCC",
                              "bllip-xs-gptbpe"="#CCCCCC")) +
    theme(axis.text=element_text(size=14),
          axis.text.y = element_text(size = 10),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "right")
ggsave("../images/cuny2020/surp_corr.png",height=4.5,width=12)

LS0tCnRpdGxlOiAiQ1VOWSAyMDIwIEFuYWx5c2lzIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgojIFBhY2thZ2VzIGFuZCB1dGlsaXRpZXMKCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShsbWU0KQpsaWJyYXJ5KGxtZXJUZXN0KQpsaWJyYXJ5KHBsb3RyaXgpCmxpYnJhcnkoc3RyaW5ncikKbGlicmFyeShyZWFkeGwpCmxpYnJhcnkoUkNvbG9yQnJld2VyKQpsaWJyYXJ5KG12dG5vcm0pCmxpYnJhcnkobWdjdikKYGBgCgpgYGB7cn0KIyBDb21wdXRlIHRoZSBsb2ctbGlrZWxpaG9vZCBvZiBhIG5ldyBkYXRhc2V0IHVzaW5nIGEgZml0IGxtZTQgbW9kZWwuCmxvZ0xpa190ZXN0IDwtIGZ1bmN0aW9uKGxtLCB0ZXN0X1gsIHRlc3RfeSkgewogIHByZWRpY3Rpb25zIDwtIHByZWRpY3QobG0sIHRlc3RfWCwgcmUuZm9ybT1OQSkKICAjIEdldCBzdGQuZGV2LiBvZiByZXNpZHVhbCwgZXN0aW1hdGVkIGZyb20gdHJhaW4gZGF0YQogIHN0ZGV2IDwtIHNpZ21hKGxtKQogICMgRm9yIGVhY2ggcHJlZGljdGlvbi0tb2JzZXJ2YXRpb24sIGdldCB0aGUgZGVuc2l0eSBwKG9icyB8IE4ocHJlZGljdGVkLCBtb2RlbF9zaWdtYSkpIGFuZCByZWR1Y2UKICBkZW5zaXR5IDwtIHN1bShkbm9ybSh0ZXN0X3ksIHByZWRpY3Rpb25zLCBzdGRldiwgbG9nPVRSVUUpKQogIHJldHVybihkZW5zaXR5KQp9CiMgR2V0IHBlci1wcmVkaWN0aW9uIGxvZy1saWtlbGlob29kCmxvZ0xpa190ZXN0X3BlciA8LSBmdW5jdGlvbihsbSwgdGVzdF9YLCB0ZXN0X3kpIHsKICBwcmVkaWN0aW9ucyA8LSBwcmVkaWN0KGxtLCB0ZXN0X1gsIHJlLmZvcm09TkEpCiAgIyBHZXQgc3RkLmRldi4gb2YgcmVzaWR1YWwsIGVzdGltYXRlZCBmcm9tIHRyYWluIGRhdGEKICBzdGRldiA8LSBzaWdtYShsbSkKICAjIEZvciBlYWNoIHByZWRpY3Rpb24tLW9ic2VydmF0aW9uLCBnZXQgdGhlIGRlbnNpdHkgcChvYnMgfCBOKHByZWRpY3RlZCwgbW9kZWxfc2lnbWEpKQogIGRlbnNpdGllcyA8LSBkbm9ybSh0ZXN0X3ksIHByZWRpY3Rpb25zLCBzdGRldiwgbG9nPVRSVUUpCiAgcmV0dXJuKGRlbnNpdGllcykKfQojIENvbXB1dGUgTVNFIG9mIGEgbmV3IGRhdGFzZXQgdXNpbmcgYSBmaXQgbG1lNCBtb2RlbC4KbXNlX3Rlc3QgPC0gZnVuY3Rpb24obG0sIHRlc3RfWCwgdGVzdF95KSB7CiAgcmV0dXJuKG1lYW4oKHByZWRpY3QobG0sIHRlc3RfWCwgcmUuZm9ybT1OQSkgLSB0ZXN0X3kpIF4gMikpCn0KI1Nhbml0eSBjaGVja3MKI215bG0gPC0gZ2FtKHBzeWNob21ldHJpYyB+ICBzKHN1cnByaXNhbCwgYnMgPSAiY3IiLCBrID0gMjApICsgcyhwcmV2X3N1cnAsIGJzID0gImNyIiwgayA9IDIwKSArIHRlKGZyZXEsIGxlbiwgYnMgPSAiY3IiKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gImNyIiksIGRhdGE9dHJhaW5fZGF0YSkKI2MobG9nTGlrKG15bG0pLCBsb2dMaWtfdGVzdChteWxtLCB0cmFpbl9kYXRhLCB0cmFpbl9kYXRhJHBzeWNob21ldHJpYykpCiNsb2dMaWtfdGVzdChteWxtLCB0ZXN0X2RhdGEsIHRlc3RfZGF0YSRwc3ljaG9tZXRyaWMpCmBgYAoKIyBEYXRhIGxvYWRpbmcgYW5kIHByZXByb2Nlc3NpbmcKCmBgYHtyfQpkYXRhID0gcmVhZC5jc3YoIi4uL2RhdGEvaGFybW9uaXplZF9yZXN1bHRzLmNzdiIpCgphbGxfZGF0YSA9IGRhdGEgJT4lCiAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCkpICU+JQogIGdyb3VwX2J5KGNvcnB1cywgbW9kZWwsIHRyYWluaW5nLCBzZWVkKSAlPiUKICAgIG11dGF0ZShwcmV2X3N1cnAgPSBsYWcoc3VycHJpc2FsKSwKICAgICAgICAgcHJldl9jb2RlID0gbGFnKGNvZGUpLAogICAgICAgICBwcmV2X2xlbiA9IGxhZyhsZW4pLAogICAgICAgICBwcmV2X2ZyZXEgPSBsYWcoZnJlcSksCiAgICAgICAgIHByZXZfc3VycCA9IGxhZyhzdXJwcmlzYWwpLAogICAgICAgICAKICAgICAgICAgcHJldjJfZnJlcSA9IGxhZyhwcmV2X2ZyZXEpLAogICAgICAgICBwcmV2Ml9jb2RlID0gbGFnKHByZXZfY29kZSksCiAgICAgICAgIHByZXYyX2xlbiA9IGxhZyhwcmV2X2xlbiksCiAgICAgICAgIHByZXYyX3N1cnAgPSBsYWcocHJldl9zdXJwKSwKICAgICAgICAgCiAgICAgICAgIHByZXYzX2ZyZXEgPSBsYWcocHJldjJfZnJlcSksCiAgICAgICAgIHByZXYzX2NvZGUgPSBsYWcocHJldjJfY29kZSksCiAgICAgICAgIHByZXYzX2xlbiA9IGxhZyhwcmV2Ml9sZW4pLAogICAgICAgICBwcmV2M19zdXJwID0gbGFnKHByZXYyX3N1cnApKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgIyBGaWx0ZXIgYmFjayB0aHJlZSBmb3IgdGhlIGR1bmRlZSBjb3JwdXMuIEZpbHRlciBiYWNrIDEgZm9yIGFsbCBvdGhlciBjb3Jwb3JhCiAgZmlsdGVyKChjb3JwdXMgPT0gImR1bmRlZSIgJiBjb2RlID09IHByZXYyX2NvZGUgKyAyKSB8IChjb3JwdXMgIT0gImR1bmRlZSIgJiBjb2RlID09IHByZXZfY29kZSArIDEpKSAlPiUKICBzZWxlY3QoLXByZXZfY29kZSwgLXByZXYyX2NvZGUsIC1wcmV2M19jb2RlKSAlPiUKICBkcm9wX25hKCkKCmFsbF9kYXRhID0gYWxsX2RhdGEgJT4lCiAgbXV0YXRlKAogICAgbW9kZWwgPSBhcy5jaGFyYWN0ZXIobW9kZWwpLAogICAgbW9kZWwgPSBpZl9lbHNlKG1vZGVsID09ICJncHQtMiIsICJncHQyIiwgbW9kZWwpLAogICAgbW9kZWwgPSBhcy5mYWN0b3IobW9kZWwpKQogIApgYGAKCgogCmBgYHtyfQojIENvbXB1dGUgbGluZWFyIG1vZGVsIHN0YXRzIGZvciB0aGUgZ2l2ZW4gdHJhaW5pbmcgZGF0YSBzdWJzZXQgYW5kIGZ1bGwgdGVzdCBkYXRhLgojIEF1dG9tYXRpY2FsbHkgc3Vic2V0cyB0aGUgdGVzdCBkYXRhIHRvIG1hdGNoIHRoZSByZWxldmFudCBncm91cCBmb3Igd2hpY2ggd2UgYXJlIHRyYWluaW5nIGEgbGluZWFyIG1vZGVsLgpnZXRfbG1fZGF0YSA8LSBmdW5jdGlvbihkZiwgdGVzdF9kYXRhLCBmb3JtdWxhLCBzdG9yZV9lbnYpIHsKICAjdGhpc19sbSA8LSBnYW0oZm9ybXVsYSwgZGF0YT1kZik7CiAgdGhpc19sbSA9IGxtKGZvcm11bGEsIGRhdGE9ZGYpCiAgdGhpc190ZXN0X2RhdGEgPC0gc2VtaV9qb2luKHRlc3RfZGF0YSwgZGYsIGJ5PWMoInRyYWluaW5nIiwgIm1vZGVsIiwgInNlZWQiLCAiY29ycHVzIikpOwogIAogICMgU2F2ZSBsbSB0byB0aGUgZ2xvYmFsIGVudiBzbyB0aGF0IHdlIGNhbiBhY2Nlc3MgcmVzaWR1YWxzIGxhdGVyLgogIGxtX25hbWUgPSB1bmlxdWUocGFzdGUoZGYkbW9kZWwsIGRmJHRyYWluaW5nLCBkZiRzZWVkLCBkZiRjb3JwdXMpKVsxXQogIGFzc2lnbihsbV9uYW1lLCB0aGlzX2xtLCBlbnZpcj1zdG9yZV9lbnYpCiAgCiAgc3VtbWFyaXNlKGRmLAogICAgICAgICAgICBsb2dfbGlrID0gYXMubnVtZXJpYyhsb2dMaWsodGhpc19sbSwgUkVNTCA9IEYpKSwKICAgICAgICAgICAgdGVzdF9saWsgPSBsb2dMaWtfdGVzdCh0aGlzX2xtLCB0aGlzX3Rlc3RfZGF0YSwgdGhpc190ZXN0X2RhdGEkcHN5Y2hvbWV0cmljKSwKICAgICAgICAgICAgdGVzdF9tc2UgPSBtc2VfdGVzdCh0aGlzX2xtLCB0aGlzX3Rlc3RfZGF0YSwgdGhpc190ZXN0X2RhdGEkcHN5Y2hvbWV0cmljKSkKfQojIEZvciBhIHByZXZpb3VzbHkgZml0dGVkIGxtIHN0b3JlZCBpbiBzdG9yZV9lbnYsIGdldCB0aGUgcmVzaWR1YWxzIG9uIHRlc3QgZGF0YSBvZiB0aGUgcmVsZXZhbnQgZGF0YSBzdWJzZXQuCmdldF9sbV9yZXNpZHVhbHMgPC0gZnVuY3Rpb24oZGYsIHN0b3JlX2VudikgewogICMgUmV0cmlldmUgdGhlIHJlbGV2YW50IGxtLgogIGxtX25hbWUgPSB1bmlxdWUocGFzdGUoZGYkbW9kZWwsIGRmJHRyYWluaW5nLCBkZiRzZWVkLCBkZiRjb3JwdXMpKVsxXQogIHRoaXNfbG0gPC0gZ2V0KGxtX25hbWUsIGVudmlyPXN0b3JlX2VudikKICAKICBtdXRhdGUoZGYsCiAgICAgICAgIGxpa2VsaWhvb2QgPSBsb2dMaWtfdGVzdF9wZXIodGhpc19sbSwgZGYsIGRmJHBzeWNob21ldHJpYyksCiAgICAgICAgIHJlc2lkID0gZGYkcHN5Y2hvbWV0cmljIC0gcHJlZGljdCh0aGlzX2xtLCBkZiwgcmUuZm9ybT1OQSkpCn0KIyMjIyMKIyBEZWZpbmUgcmVncmVzc2lvbiBmb3JtdWxhZS4KIyBFeWUtdHJhY2tpbmcgcmVncmVzc2lvbjogb25seSB1c2Ugc3VycHJpc2FsIGFuZCBwcmV2aW91cyBzdXJwcmlzYWw7IFNQUlQgcmVncmVzc2lvbjogdXNlIDItYmFjayBmZWF0dXJlcy4KI2Jhc2VsaW5lX3J0X3JlZ3Jlc3Npb24gPSBwc3ljaG9tZXRyaWMgfiB0ZShmcmVxLCBsZW4sIGJzID0gImNyIikgKyB0ZShwcmV2X2ZyZXEsIHByZXZfbGVuLCBicyA9ICJjciIpCiNiYXNlbGllX3NwcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IHRlKGZyZXEsIGxlbiwgYnMgPSAiY3IiKSArIHRlKHByZXZfZnJlcSwgcHJldl9sZW4sIGJzID0gImNyIikgKyB0ZShwcmV2Ml9mcmVxLCBwcmV2Ml9sZW4sIGJzID0gImNyIikKI2Z1bGxfcnRfcmVncmVzc2lvbiA9IChwc3ljaG9tZXRyaWMgfiBzKHN1cnByaXNhbCwgYnMgPSAiY3IiLCBrID0gMjApICsgcyhwcmV2X3N1cnAsIGJzID0gImNyIiwgayA9IDIwKQogICAgICAgICAgICAgICAgICAgICAjKyB0ZShmcmVxLCBsZW4sIGJzID0gImNyIikgKyB0ZShwcmV2X2ZyZXEsIHByZXZfbGVuLCBicyA9ICJjciIpKQojZnVsbF9zcHJ0X3JlZ3Jlc3Npb24gPSAocHN5Y2hvbWV0cmljIH4gcyhzdXJwcmlzYWwsIGJzID0gImNyIiwgayA9IDIwKSArIHMocHJldl9zdXJwLCBicyA9ICJjciIsIGsgPSAyMCkgKyBzKHByZXYyX3N1cnAsIGJzID0gImNyIiwgayA9IDIwKQogICAgICAgICAgICAgICAgICAgICAgICAjKyB0ZShmcmVxLCBsZW4sIGJzID0gImNyIikgKyB0ZShwcmV2X2ZyZXEsIHByZXZfbGVuLCBicyA9ICJjciIpICsgdGUocHJldjJfZnJlcSwgcHJldjJfbGVuLCBicyA9ICJjciIpKQoKYmFzZWxpbmVfcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IGZyZXEgKyBwcmV2X2ZyZXEgKyBwcmV2Ml9mcmVxICsgcHJldjNfZnJlcSArIGxlbiArIHByZXZfbGVuICsgcHJldjJfbGVuICsgcHJldjNfbGVuCmJhc2VsaW5lX3NwcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IGZyZXEgKyBwcmV2X2ZyZXEgKyBsZW4gKyBwcmV2X2xlbgoKZnVsbF9ydF9yZWdyZXNzaW9uID0gcHN5Y2hvbWV0cmljIH4gc3VycHJpc2FsICsgcHJldl9zdXJwICsgcHJldjJfc3VycCArIHByZXYzX3N1cnAgKyBmcmVxICsgcHJldl9mcmVxICsgcHJldjJfZnJlcSArIHByZXYzX2ZyZXEgKyBsZW4gKyBwcmV2X2xlbiArIHByZXYyX2xlbiArIHByZXYzX2xlbgpmdWxsX3NwcnRfcmVncmVzc2lvbiA9IHBzeWNob21ldHJpYyB+IHN1cnByaXNhbCArIHByZXZfc3VycCArIGZyZXEgKyBwcmV2X2ZyZXEgKyBsZW4gKyBwcmV2X2xlbgogIAojIyMjIwojIFByZXBhcmUgZnJhbWVzL2Vudmlyb25tZW50cyBmb3Igc3RvcmluZyByZXN1bHRzL29iamVjdHMuCmJhc2VsaW5lX3Jlc3VsdHMgPSBkYXRhLmZyYW1lKCkKZnVsbF9tb2RlbF9yZXN1bHRzID0gZGF0YS5mcmFtZSgpCmJhc2VsaW5lX3Jlc2lkdWFscyA9IGRhdGEuZnJhbWUoKQpmdWxsX3Jlc2lkdWFscyA9IGRhdGEuZnJhbWUoKQojUmFuZG9tbHkgc2h1ZmZsZSB0aGUgZGF0YQphbGxfZGF0YTwtYWxsX2RhdGFbc2FtcGxlKG5yb3coYWxsX2RhdGEpKSxdCiNDcmVhdGUgSyBlcXVhbGx5IHNpemUgZm9sZHMKSyA9IDUKZm9sZHMgPC0gY3V0KHNlcSgxLG5yb3coYWxsX2RhdGEpKSxicmVha3M9SyxsYWJlbHM9RkFMU0UpCiNQZXJmb3JtIDEwIGZvbGQgY3Jvc3MgdmFsaWRhdGlvbgpiYXNlbGluZV9jb3JwdXMgPSBmdW5jdGlvbihjb3JwdXMsIGRmLCB0ZXN0X2RhdGEsIGVudikgewogIGlmKGNvcnB1cyA9PSAiZHVuZGVlIikgewogICAgZ2V0X2xtX2RhdGEoZGYsIHRlc3RfZGF0YSwgYmFzZWxpbmVfcnRfcmVncmVzc2lvbiwgZW52KQogIH0gZWxzZSB7CiAgICBnZXRfbG1fZGF0YShkZiwgdGVzdF9kYXRhLCBiYXNlbGluZV9zcHJ0X3JlZ3Jlc3Npb24sIGVudikKICB9Cn0KZnVsbF9tb2RlbF9jb3JwdXMgPSBmdW5jdGlvbihjb3JwdXMsIGRmLCB0ZXN0X2RhdGEsIGVudikgewogIGlmKGNvcnB1c1sxXSA9PSAiZHVuZGVlIikgewogICAgZ2V0X2xtX2RhdGEoZGYsIHRlc3RfZGF0YSwgZnVsbF9ydF9yZWdyZXNzaW9uLCBlbnYpCiAgfSBlbHNlIHsKICAgIGdldF9sbV9kYXRhKGRmLCB0ZXN0X2RhdGEsIGZ1bGxfc3BydF9yZWdyZXNzaW9uLCBlbnYpCiAgfQp9CmZvcihpIGluIDE6SykgeyAKICAjU2VnZW1lbnQgeW91ciBkYXRhIGJ5IGZvbGQgdXNpbmcgdGhlIHdoaWNoKCkgZnVuY3Rpb24gCiAgdGVzdEluZGV4ZXMgPC0gd2hpY2goZm9sZHM9PWksYXJyLmluZD1UUlVFKQogIHRlc3RfZGF0YSA8LSBhbGxfZGF0YVt0ZXN0SW5kZXhlcywgXQogIHRyYWluX2RhdGEgPC0gYWxsX2RhdGFbLXRlc3RJbmRleGVzLCBdCiAgCiAgIyBQcmVwYXJlIGEgbmV3IEVudmlyb25tZW50IGluIHdoaWNoIHdlIHN0b3JlIGZpdHRlZCBMTXMsIHdoaWNoIHdlJ2xsIHF1ZXJ5IGxhdGVyIGZvciByZXNpZHVhbHMuCiAgYmFzZWxpbmVfZW52ID0gZW52aXJvbm1lbnQoKQogIGZ1bGxfZW52ID0gZW52aXJvbm1lbnQoKQogIAogICMgQ29tcHV0ZSBhIGJhc2VsaW5lIGxpbmVhciBtb2RlbCBmb3IgZWFjaCBtb2RlbC0tdHJhaW5pbmctLXNlZWQtLVJULWNvcnB1cyBjb21iaW5hdGlvbi4KICBiYXNlbGluZXMgPSB0cmFpbl9kYXRhICU+JQogICAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JQogICAgICBwcmludChtb2RlbCkgJT4lCiAgICAgIGRvKGJhc2VsaW5lX2NvcnB1cyh1bmlxdWUoLiRjb3JwdXMpLCAuLCB0ZXN0X2RhdGEsIGJhc2VsaW5lX2VudikpICU+JQogICAgdW5ncm91cCgpICU+JQogICAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCksCiAgICAgICAgICAgZm9sZCA9IGkpCiAgCiAgYmFzZWxpbmVfcmVzdWx0cyA9IHJiaW5kKGJhc2VsaW5lX3Jlc3VsdHMsIGJhc2VsaW5lcykKICAKICAjIENvbXB1dGUgYSBmdWxsIGxpbmVhciBtb2RlbCBmb3IgZWFjaCBtb2RlbC0tdHJhaW5pbmctLXNlZWQtUlQtY29ycHVzIGNvbWJpbmF0aW9uCiAgZnVsbF9tb2RlbHMgPSB0cmFpbl9kYXRhICU+JQogICAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JQogICAgICBkbyhmdWxsX21vZGVsX2NvcnB1cyh1bmlxdWUoLiRjb3JwdXMpLCAuLCB0ZXN0X2RhdGEsIGZ1bGxfZW52KSkgJT4lCiAgICB1bmdyb3VwKCkgJT4lCiAgICBtdXRhdGUoc2VlZCA9IGFzLmZhY3RvcihzZWVkKSwKICAgICAgICAgICBmb2xkID0gaSkKICAKICBmdWxsX21vZGVsX3Jlc3VsdHMgPSByYmluZChmdWxsX21vZGVsX3Jlc3VsdHMsIGZ1bGxfbW9kZWxzKQogIAogIGZvbGRfYmFzZWxpbmVfcmVzaWR1YWxzID0gdGVzdF9kYXRhICU+JQogICAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JQogICAgICBkbyhnZXRfbG1fcmVzaWR1YWxzKC4sIGJhc2VsaW5lX2VudikpICU+JQogICAgdW5ncm91cCgpCiAgCiAgYmFzZWxpbmVfcmVzaWR1YWxzID0gcmJpbmQoYmFzZWxpbmVfcmVzaWR1YWxzLCBmb2xkX2Jhc2VsaW5lX3Jlc2lkdWFscykKICAKICBmb2xkX2Z1bGxfcmVzaWR1YWxzID0gdGVzdF9kYXRhICU+JQogICAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JQogICAgICBkbyhnZXRfbG1fcmVzaWR1YWxzKC4sIGZ1bGxfZW52KSkgJT4lCiAgICB1bmdyb3VwKCkgCiAgCiAgZnVsbF9yZXNpZHVhbHMgPSByYmluZChmdWxsX3Jlc2lkdWFscywgZm9sZF9mdWxsX3Jlc2lkdWFscykKfQpgYGAKCmBgYHtyfQp3cml0ZS5jc3YoZnVsbF9yZXNpZHVhbHMsICIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2Z1bGxfcmVzaWR1YWxzLmNzdiIpCndyaXRlLmNzdihiYXNlbGluZV9yZXNpZHVhbHMsICIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2Jhc2VsaW5lX3Jlc2lkdWFscy5jc3YiKQpgYGAKCmBgYHtyfQp3cml0ZS5jc3YoZnVsbF9tb2RlbF9yZXN1bHRzLCAiLi4vZGF0YS9hbmFseXNpc19jaGVja3BvaW50cy9mdWxsX21vZGVsX3Jlc3VsdC5jc3YiKQp3cml0ZS5jc3YoYmFzZWxpbmVfcmVzdWx0cywgIi4uL2RhdGEvYW5hbHlzaXNfY2hlY2twb2ludHMvYmFzZWxpbmVfcmVzdWx0cy5jc3YiKQojZnVsbF9tb2RlbF9yZXN1bHRzID0gcmVhZC5jc3YoIi4uL2RhdGEvYW5hbHlzaXNfY2hlY2twb2ludHMvZmZ1bGxfbW9kZWxfcmVzdWx0cy5jc3YiKQojYmFzZWxpbmVfcmVzdWx0cyA9IHJlYWQuY3N2KCIuLi9kYXRhL2FuYWx5c2lzX2NoZWNrcG9pbnRzL2ZiYXNlbGluZV9yZXN1bHRzYi5jc3YiKQoKIyBKb2luIGJhc2VsaW5lIG1vZGVscyB3aXRoIGZ1bGwgbW9kZWxzIGFuZCBjb21wYXJlIHBlcmZvcm1hbmNlIHdpdGhpbi1mb2xkLgptb2RlbF9mb2xkX2RlbHRhcyA9IGJhc2VsaW5lX3Jlc3VsdHMgJT4lCiAgcmlnaHRfam9pbihmdWxsX21vZGVsX3Jlc3VsdHMsIHN1ZmZpeD1jKCIuYmFzZWxpbmUiLCAiLmZ1bGwiKSwKICAgICAgICAgICAgIGJ5PWMoIm1vZGVsIiwgInRyYWluaW5nIiwgInNlZWQiLCAiY29ycHVzIiwgImZvbGQiKSkgJT4lCiAgCiAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCkpICU+JQogIAogICMgQ29tcHV0ZSBwZXItZm9sZCBkZWx0YXMuCiAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMsIGZvbGQpICU+JQogICAgbXV0YXRlKGRlbHRhX2xvZ19saWsgPSB0ZXN0X2xpay5mdWxsIC0gdGVzdF9saWsuYmFzZWxpbmUsCiAgICAgICAgICAgZGVsdGFfbXNlID0gdGVzdF9tc2UuZnVsbCAtIHRlc3RfbXNlLmJhc2VsaW5lKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgc2VsZWN0KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzLCBmb2xkLAogICAgICAgICBkZWx0YV9sb2dfbGlrLCBkZWx0YV9tc2UpCgojIE5vdyBjb21wdXRlIGFjcm9zcy1mb2xkIGRlbHRhIHN0YXRpc3RpY3MgZm9yIGVhY2ggbW9kZWwtLXRyYWluaW5nLS1zZWVkLS1jb3JwdXMuCm1vZGVsX2RlbHRhcyA9IG1vZGVsX2ZvbGRfZGVsdGFzICU+JQogIGdyb3VwX2J5KG1vZGVsLCB0cmFpbmluZywgc2VlZCwgY29ycHVzKSAlPiUKICAgIHN1bW1hcmlzZShtZWFuX2RlbHRhX2xvZ19saWsgPSBzdW0oZGVsdGFfbG9nX2xpayksCiAgICAgICAgICAgICAgc2VtX2RlbHRhX2xvZ19saWsgPSBzZChkZWx0YV9sb2dfbGlrKSAvIHNxcnQobGVuZ3RoKGRlbHRhX2xvZ19saWspKSwKICAgICAgICAgICAgICBtZWFuX2RlbHRhX21zZSA9IHN1bShkZWx0YV9tc2UpLAogICAgICAgICAgICAgIHNlbV9kZWx0YV9tc2UgPSBzZChkZWx0YV9tc2UpIC8gc3FydChsZW5ndGgoZGVsdGFfbXNlKSkpCmBgYAoKYGBge3J9Cm1ldHJpYyA8LSAizpRMb2dMaWsiCiNtZXRyaWMgPC0gIi3OlE1TRSIKIyBTZWxlY3QgdGhlIHJlbGV2YW50IG1ldHJpYy4KbW9kZWxfZm9sZF9kZWx0YXMgPSBtb2RlbF9mb2xkX2RlbHRhcyAlPiUKICAjIFJldHJpZXZlIHRoZSBjdXJyZW50IHRlc3QgbWV0cmljCiAgbXV0YXRlKGRlbHRhX3Rlc3QgPSBkZWx0YV9sb2dfbGlrKSAlPiUKICBzZWxlY3QoLWRlbHRhX2xvZ19saWssIC1kZWx0YV9tc2UpCiMgU2VsZWN0IHRoZSByZWxldmFudCBtZXRyaWMuCm1vZGVsX2RlbHRhcyA9IG1vZGVsX2RlbHRhcyAlPiUKICAgICMgUmV0cmlldmUgdGhlIGN1cnJlbnQgdGVzdCBtZXRyaWMKICAgIG11dGF0ZShkZWx0YV90ZXN0X21lYW4gPSBtZWFuX2RlbHRhX2xvZ19saWssCiAgICAgICAgICAgZGVsdGFfdGVzdF9zZW0gPSBzZW1fZGVsdGFfbG9nX2xpaykgJT4lCiAgICAjIG11dGF0ZShkZWx0YV90ZXN0X21lYW4gPSBtZWFuX2RlbHRhX21zZSwKICAgICMgICAgICAgIGRlbHRhX3Rlc3Rfc2VtID0gc2VtX2RlbHRhX21zZSkKICAgIAogICAgIyBSZW1vdmUgdGhlIHJhdyBtZXRyaWNzLgogICAgc2VsZWN0KC1tZWFuX2RlbHRhX2xvZ19saWssIC1zZW1fZGVsdGFfbG9nX2xpaywKICAgICAgICAgICAtbWVhbl9kZWx0YV9tc2UsIC1zZW1fZGVsdGFfbXNlKQptb2RlbF9kZWx0YXMKYGBgCgpgYGB7cn0KIyBTYW5pdHkgY2hlY2s6IHRyYWluaW5nIG9uIHRyYWluK3Rlc3QgZGF0YSBzaG91bGQgeWllbGQgaW1wcm92ZWQgcGVyZm9ybWFuY2Ugb3ZlciB0cmFpbmluZyBvbiBqdXN0IHRyYWluaW5nIGRhdGEuIChXaGVuIGV2YWx1YXRpbmcgb24gdGVzdCBkYXRhLikKIyBmdWxsX2Jhc2VsaW5lcyA9IGFsbF9kYXRhICU+JQojICAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBzZWVkLCBjb3JwdXMpICU+JQojICAgc3VtbWFyaXNlKGJhc2VsaW5lX3RyYWluX2FsbF90ZXN0X2xpayA9IGxvZ0xpa190ZXN0KGxtKHBzeWNob21ldHJpYyB+IGxlbiArIGZyZXEgKyBzZW50X3BvcywgZGF0YT0uKSwgc2VtaV9qb2luKHRlc3RfZGF0YSwgLiwgYnk9YygidHJhaW5pbmciLCAibW9kZWwiLCAic2VlZCIsICJjb3JwdXMiKSksIHNlbWlfam9pbih0ZXN0X2RhdGEsIC4sIGJ5PWMoInRyYWluaW5nIiwgIm1vZGVsIiwgInNlZWQiLCAiY29ycHVzIikpJHBzeWNob21ldHJpYykpICU+JQojICAgdW5ncm91cCgpCiMgZnVsbF9iYXNlbGluZXMKIyAKIyBmdWxsX2Jhc2VsaW5lcyAlPiUKIyAgIHJpZ2h0X2pvaW4oYmFzZWxpbmVzLCBieT1jKCJzZWVkIiwgInRyYWluaW5nIiwgIm1vZGVsIiwgImNvcnB1cyIpKSAlPiUKIyAgIG11dGF0ZShkZWx0YT1iYXNlbGluZV90cmFpbl9hbGxfdGVzdF9saWstYmFzZWxpbmVfdGVzdF9saWspICU+JQojICAgc2VsZWN0KC1iYXNlbGluZV9saWspICMgJT4lCiMgICAjc2VsZWN0KC1iYXNlbGluZV90ZXN0X2xpaywgLWJhc2VsaW5lX3RyYWluX2FsbF90ZXN0X2xpaywgLWJhc2VsaW5lX2xpaywgLWJhc2VsaW5lX3Rlc3RfbXNlKQpgYGAKCiMgTG9hZCBsYW5ndWFnZSBtb2RlbCBkYXRhIChTeW50YXhHeW0sIFBQTCkKCmBgYHtyfQpsYW5ndWFnZV9tb2RlbF9kYXRhID0gcmVhZC5jc3YoIi4uL2RhdGEvbW9kZWxfbWV0YWRhdGEuY3N2IikgJT4lCiAgbXV0YXRlKG1vZGVsID0gYXMuY2hhcmFjdGVyKG1vZGVsKSwKICAgICAgICAgbW9kZWwgPSBpZl9lbHNlKG1vZGVsID09ICJncHQtMiIsICJncHQyIiwgbW9kZWwpLAogICAgICAgICBtb2RlbCA9IGFzLmZhY3Rvcihtb2RlbCkpICU+JQogIG11dGF0ZSh0cmFpbl9zaXplID0gY2FzZV93aGVuKHN0cl9zdGFydHModHJhaW5pbmcsICJibGxpcC1sZyIpIH4gNDIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RyX3N0YXJ0cyh0cmFpbmluZywgImJsbGlwLW1kIikgfiAxNSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdHJfc3RhcnRzKHRyYWluaW5nLCAiYmxsaXAtc20iKSB+IDUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RyX3N0YXJ0cyh0cmFpbmluZywgImJsbGlwLXhzIikgfiAxKSkgJT4lCiAgbXV0YXRlKHNlZWQgPSBhcy5mYWN0b3Ioc2VlZCkpICU+JQogIHNlbGVjdCgtcGlkLCAtdGVzdF9sb3NzKSAlPiUKICBkaXN0aW5jdChtb2RlbCwgdHJhaW5pbmcsIHNlZWQsIC5rZWVwX2FsbCA9IFRSVUUpCnRhYmxlKGxhbmd1YWdlX21vZGVsX2RhdGEkc2VlZCkKdGFibGUobW9kZWxfZGVsdGFzJHNlZWQpCmBgYAoKRmlyc3Qgam9pbiBkZWx0YS1tZXRyaWMgZGF0YSB3aXRoIG1vZGVsIGF1eGlsaWFyeSBkYXRhLgoKYGBge3J9Cm1vZGVsX2RlbHRhcyA9IG1vZGVsX2RlbHRhcyAlPiUKICBtZXJnZShsYW5ndWFnZV9tb2RlbF9kYXRhLCBieSA9IGMoInNlZWQiLCAidHJhaW5pbmciLCAibW9kZWwiKSwgYWxsPVQpICU+JQogIGRyb3BfbmEoKQoKbW9kZWxfZm9sZF9kZWx0YXMgPSBtb2RlbF9mb2xkX2RlbHRhcyAlPiUKICBtZXJnZShsYW5ndWFnZV9tb2RlbF9kYXRhLCBieSA9IGMoInNlZWQiLCAidHJhaW5pbmciLCAibW9kZWwiKSwgYWxsPVQpICU+JQogIGRyb3BfbmEoKQoKbW9kZWxfZGVsdGFzCmBgYAoKQWxzbyBqb2luIG9uIHRoZSBvcmlnaW5hbCBsaW5lYXIgbW9kZWwgZGF0YSwgcmF0aGVyIHRoYW4gY29sbGFwc2luZyB0byBkZWx0YS1tZXRyaWNzLgpUaGlzIHdpbGwgc3VwcG9ydCByZWdyZXNzaW9ucyBsYXRlciBvbiB0aGF0IGRvbid0IGNvbGxhcHNlIGFjcm9zcyBmb2xkcy4KCgojIEZpbmFsIGRhdGEgcHJlcHJvY2Vzc2luZwoKYGBge3IgRmlsdGVyIG1vZGVscyBhbmQvb3IgY29ycG9yYX0KIyBFeGNsdWRlIG9yZGVyZWQtbmV1cm9ucyBmcm9tIGFsbCBhbmFseXNlcy4KbW9kZWxfZGVsdGFzIDwtIG1vZGVsX2RlbHRhcyAlPiUKICBmaWx0ZXIobW9kZWwgIT0gIm9yZGVyZWQtbmV1cm9ucyIpCm1vZGVsX2ZvbGRfZGVsdGFzIDwtIG1vZGVsX2ZvbGRfZGVsdGFzICU+JQogIGZpbHRlcihtb2RlbCAhPSAib3JkZXJlZC1uZXVyb25zIikKYGBgCgoKIyBWaXN1YWxpemF0aW9ucwoKIyMgUHJlZGljdGl2ZSBwb3dlciBhbmQgU0cKCgpgYGB7ciBCeSBtb2RlbH0KbW9kZWxfZGVsdGFzICU+JQogIGdncGxvdChhZXMoeD1zZ19zY29yZSwgeT1kZWx0YV90ZXN0X21lYW4pKSArCiAgICBnZW9tX2Vycm9yYmFyKGFlcyh5bWluPWRlbHRhX3Rlc3RfbWVhbi1kZWx0YV90ZXN0X3NlbSwgeW1heD1kZWx0YV90ZXN0X21lYW4rZGVsdGFfdGVzdF9zZW0pKSArCiAgICBnZW9tX3Ntb290aChtZXRob2Q9ImxtIiwgc2U9VCkgKwogICAgZ2VvbV9wb2ludChzdGF0PSJpZGVudGl0eSIsIHBvc2l0aW9uPSJkb2RnZSIsIGFscGhhPTEsIHNpemU9MywgYWVzKGNvbG9yPXRyYWluaW5nLCBzaGFwZT1tb2RlbCkpICsKICAgIHlsYWIobWV0cmljKSArCiAgICB4bGFiKCJTeW50YXggR2VuZXJhbGl6YXRpb24gU2NvcmUiKSArCiAgICBnZ3RpdGxlKCJTeW50YWN0aWMgR2VuZXJhbGl6YXRpb24gdnMuIFByZWRpY3RpdmUgUG93ZXIiKSArCiAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtbWQiPSIjMzk1NjhDRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20iPSIjMUY5NjhCRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtbGctZ3B0YnBlIj0iIzg4ODg4OCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZC1ncHRicGUiPSIjQUFBQUFBIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXNtLWdwdGJwZSI9IiNDQ0NDQ0MiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMtZ3B0YnBlIj0iI0NDQ0NDQyIpKSArCiAgICBmYWNldF9ncmlkKH5jb3JwdXMsIHNjYWxlcz0iZnJlZSIpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBzdHJpcC50ZXh0LnggPSBlbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBsZWdlbmQudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBheGlzLnRpdGxlPWVsZW1lbnRfdGV4dChzaXplPTE4KSwKICAgICAgICAgIGxlZ2VuZC5wb3NpdGlvbiA9ICJib3R0b20iKQojZ2dzYXZlKCIuL2NvZ3NjaV9pbWFnZXMvc2dfbG9nbGlrLnBuZyIsaGVpZ2h0PTUsd2lkdGg9NikKYGBgCgojIyMgUmVncmVzc2lvbiBhbmFseXNlcwoKV2UgY29udHJvbCBmb3IgZWZmZWN0cyBvZiBwZXJwbGV4aXR5IGJ5IHJlbGF0aW5nIHRoZSByZXNpZHVhbHMgb2YgYSBgcGVyZm9ybWFuY2UgfiBQUExgIHJlZ3Jlc3Npb24gdG8gU0cgc2NvcmUuCgpgYGB7ciBSZXNpZHVhbGl6ZWQgcmVncmVzc2lvbn0KIyBQcmVwYXJlIGEgcmVzaWR1YWxpemVkIHJlZ3Jlc3Npb24gZm9yIHgxIG9udG8geSwgY29udHJvbGxpbmcgZm9yIHRoZSBlZmZlY3RzIG9mIHgyLgpkX3Jlc2lkID0gbW9kZWxfZm9sZF9kZWx0YXMgJT4lCiAgZHJvcF9uYSgpICU+JQogIAogICMgUmVzaWR1YWxpemUgZGVsdGEgbWV0cmljIHcuci50IFBQTCBmb3IgZWFjaCBtb2RlbC0tdHJhaW5pbmctLXNlZWQtLWZvbGQKICBncm91cF9ieShjb3JwdXMpICU+JQogICAgbXV0YXRlKHJlc2lkLmRlbHRhID0gcmVzaWQobG0oZGVsdGFfdGVzdCB+IHRyYWluaW5nOnRlc3RfcHBsKSkpICU+JQogIHVuZ3JvdXAoKSAlPiUKICAKICAjIFJlc2lkdWFsaXplIFNHIHNjb3JlIHcuci50LiBQUEwgZm9yIGVhY2ggdHJhaW5pbmcgZ3JvdXAKICBncm91cF9ieSh0cmFpbmluZykgJT4lCiAgICAjIE5CIG5vIG5lZWQgZm9yIHRyYWluaW5nOnBwbCBpbnRlcmFjdGlvbiwgc2luY2Ugd2UncmUgd2l0aGluLWdyb3VwLgogICAgbXV0YXRlKHJlc2lkLnNnID0gcmVzaWQobG0oc2dfc2NvcmUgfiB0ZXN0X3BwbCkpKSAlPiUKICB1bmdyb3VwKCkgJT4lCiAgIyBDb21wdXRlIHN1bW1hcnkgc3RhdGlzdGljcyBhY3Jvc3MgbW9kZWwtLXRyYWluaW5nLS1zZWVkLS1jb3JwdXMuCiAgZ3JvdXBfYnkobW9kZWwsIHRyYWluaW5nLCBjb3JwdXMsIHNlZWQpICU+JQogICAgc3VtbWFyaXNlKHJlc2lkLmRlbHRhLm1lYW4gPSBtZWFuKHJlc2lkLmRlbHRhKSwKICAgICAgICAgICAgICByZXNpZC5kZWx0YS5zZW0gPSBzZChyZXNpZC5kZWx0YSkgLyBzcXJ0KGxlbmd0aChyZXNpZC5kZWx0YSkpLAogICAgICAgICAgICAgIHJlc2lkLnNnLm1lYW4gPSBtZWFuKHJlc2lkLnNnKSwKICAgICAgICAgICAgICByZXNpZC5zZy5zZW0gPSBzZChyZXNpZC5zZykgLyBzcXJ0KGxlbmd0aChyZXNpZC5zZykpKQojIE5vdyBwbG90IHJlc2lkdWFsIHZzIFNHCmRfcmVzaWQgJT4lCiAgI2ZpbHRlcihjb3JwdXMgIT0gImJuYy1icm93biIpICU+JQogIGdncGxvdChhZXMoeD1yZXNpZC5zZy5tZWFuLCB5PXJlc2lkLmRlbHRhLm1lYW4pKSArCiAgICBnZW9tX2Vycm9yYmFyKGFlcyh4bWluPXJlc2lkLnNnLm1lYW4gLSByZXNpZC5zZy5zZW0sCiAgICAgICAgICAgICAgICAgICAgICB4bWF4PXJlc2lkLnNnLm1lYW4gKyByZXNpZC5zZy5zZW0sCiAgICAgICAgICAgICAgICAgICAgICB5bWluPXJlc2lkLmRlbHRhLm1lYW4gLSByZXNpZC5kZWx0YS5zZW0sCiAgICAgICAgICAgICAgICAgICAgICB5bWF4PXJlc2lkLmRlbHRhLm1lYW4gKyByZXNpZC5kZWx0YS5zZW0pLCBhbHBoYT0wLjMpICsKICAgIGdlb21fc21vb3RoKG1ldGhvZD0ibG0iLCBzZT1UKSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT00LCBhZXMoc2hhcGU9bW9kZWwsIGNvbG9yPXRyYWluaW5nKSkgKwogICAgeWxhYihwYXN0ZSgiUmVzaWR1YWwiLCBtZXRyaWMpKSArCiAgICB4bGFiKCJSZXNpZHVhbCBTeW50YXggR2VuZXJhbGl6YXRpb24gU2NvcmUiKSArCiAgICBnZ3RpdGxlKCJTeW50YWN0aWMgR2VuZXJhbGl6YXRpb24gdnMuIFByZWRpY3RpdmUgUG93ZXIiKSArCiAgICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiYmxsaXAtbGciPSIjNDQwMTU0RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kIj0iIzM5NTY4Q0ZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMiPSIjNzNEMDU1RkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLWxnLWdwdGJwZSI9IiM4ODg4ODgiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kLWdwdGJwZSI9IiNBQUFBQUEiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXNtLWdwdGJwZSI9IiNDQ0NDQ0MiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXhzLWdwdGJwZSI9IiNDQ0NDQ0MiKSkgKwogICAgZmFjZXRfZ3JpZCgufmNvcnB1cywgc2NhbGVzPSJmcmVlIikgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTgpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gInJpZ2h0IikKZ2dzYXZlKCIuLi9pbWFnZXMvY3VueTIwMjAvcHBsX3NnLnBuZyIsaGVpZ2h0PTQuNSx3aWR0aD0xMSkKYGBgCgoKYGBge3IgU3RlcHdpc2UgcmVncmVzc2lvbn0KZG9fc3RlcHdpc2VfcmVncmVzc2lvbiA9IGZ1bmN0aW9uKGN1cl9jb3JwdXMpIHsKICByZWdyZXNzaW9uX2RhdGEgPSBtb2RlbF9mb2xkX2RlbHRhcyAlPiUKICAgIGZpbHRlcihjb3JwdXMgPT0gY3VyX2NvcnB1cykKICAKICAjIE5CIHdlJ3JlIGluY29ycG9yYXRpbmcgdmFyaWFuY2UgYWNyb3NzIGZvbGRzIGludG8gdGhpcyByZWdyZXNzaW9uLCBnb29kIQogIHByaW50KCItLS0tLS0tLS0tLS0tLS0tLS0tLS0tIikKICBwcmludChjdXJfY29ycHVzKQogIAogIGxtMSA9IGxtKGRlbHRhX3Rlc3QgfiB0cmFpbmluZzp0ZXN0X3BwbCwgZGF0YSA9IHJlZ3Jlc3Npb25fZGF0YSkKICBsbTIgPSBsbShkZWx0YV90ZXN0IH4gdHJhaW5pbmc6dGVzdF9wcGwgKyBzZ19zY29yZSwgZGF0YSA9IHJlZ3Jlc3Npb25fZGF0YSkKICBwcmludChhbm92YShsbTEsIGxtMikpCiAgc3VtbWFyeShsbTIpCn0KI2RvX3N0ZXB3aXNlX3JlZ3Jlc3Npb24oImJuYy1icm93biIpCmRvX3N0ZXB3aXNlX3JlZ3Jlc3Npb24oImR1bmRlZSIpCmRvX3N0ZXB3aXNlX3JlZ3Jlc3Npb24oIm5hdHVyYWwtc3RvcmllcyIpCmBgYAoKIyMgUHJlZGljdGl2ZSBwb3dlciBhbmQgcGVycGxleGl0eQoKYGBge3J9Cm1vZGVsX2RlbHRhcyAlPiUKICBnZ3Bsb3QoYWVzKHg9dGVzdF9wcGwsIHk9ZGVsdGFfdGVzdF9tZWFuLCBjb2xvcj10cmFpbmluZykpICsKICAgIGdlb21fZXJyb3JiYXIoYWVzKHltaW49ZGVsdGFfdGVzdF9tZWFuLWRlbHRhX3Rlc3Rfc2VtLCB5bWF4PWRlbHRhX3Rlc3RfbWVhbitkZWx0YV90ZXN0X3NlbSksIGFscGhhPTAuNCkgKwogICAgI2dlb21fc21vb3RoKG1ldGhvZD0ibG0iLCBzZT1GKSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT00LCBhZXMoc2hhcGU9bW9kZWwpKSArCiAgICB5bGFiKG1ldHJpYykgKwogICAgeGxhYigiVGVzdCBQZXJwbGV4aXR5IikgKwogICAgI2Nvb3JkX2NhcnRlc2lhbih5bGltID0gYygxLCAxNikpICsKICAgIGdndGl0bGUoIlRlc3QgUGVycGxleGl0eSB2cy4gUHJlZGljdGl2ZSBQb3dlciIpICsKICAgIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBjKCJibGxpcC1sZyI9IiM0NDAxNTRGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtbWQiPSIjMzk1NjhDRkYiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLXNtIj0iIzFGOTY4QkZGIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cyI9IiM3M0QwNTVGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtbGctZ3B0YnBlIj0iIzg4ODg4OCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtbWQtZ3B0YnBlIj0iI0FBQUFBQSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20tZ3B0YnBlIj0iI0NDQ0NDQyIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAteHMtZ3B0YnBlIj0iI0NDQ0NDQyIpKSArCiAgICBmYWNldF9ncmlkKH5jb3JwdXMsIHNjYWxlcz0iZnJlZSIpICsKICAgICNjb29yZF9jYXJ0ZXNpYW4oeWxpbSA9IGMoMCwgMTUwKSkgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTEyKSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTIpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gInJpZ2h0IikKZ2dzYXZlKCIuLi9pbWFnZXMvY3VueTIwMjAvcHBsX2xvZ2xpay5wbmciLGhlaWdodD00LjUsd2lkdGg9MTEpCgpgYGAKCmBgYHtyfQptb2RlbF9kZWx0YXMgJT4lCiAgI2ZpbHRlcihtb2RlbCAhPSAiNWdyYW0iLCB0cmFpbmluZyAhPSAiYmxsaXAtbGciKSAlPiUKICBncm91cF9ieShtb2RlbCwgY29ycHVzKSAlPiUKICAgICNzdW1tYXJpc2UobiA9IG4oKSkKICAgIHN1bW1hcmlzZShjb3JyID0gYXMubnVtZXJpYyhjb3IudGVzdChkZWx0YV90ZXN0X21lYW4sIHRlc3RfcHBsKVs0XSksCiAgICAgICAgICAgICAgcHZhbCA9IGFzLm51bWVyaWMoY29yLnRlc3QoZGVsdGFfdGVzdF9tZWFuLCB0ZXN0X3BwbClbM10pKQpgYGAKCgojIyBFZmZlY3Qgb2YgdHJhaW5pbmcgZGF0YSBzaXplCgpgYGB7ciBPbiBwcmVkaWN0aXZlIHBvd2VyfQptb2RlbF9kZWx0YXMgJT4lCiAgbXV0YXRlKHRyYWluX3NpemUgPSBsb2codHJhaW5fc2l6ZSkpICU+JQogIGdncGxvdChhZXMoeD10cmFpbl9zaXplLCB5PWRlbHRhX3Rlc3RfbWVhbiwgY29sb3I9bW9kZWwpKSArCiAgICBnZW9tX2Vycm9yYmFyKGFlcyh5bWluPWRlbHRhX3Rlc3RfbWVhbi1kZWx0YV90ZXN0X3NlbSwgeW1heD1kZWx0YV90ZXN0X21lYW4rZGVsdGFfdGVzdF9zZW0pLCB3aWR0aCA9IDAuMSkgKwogICAgZ2VvbV9zbW9vdGgobWV0aG9kPSJsbSIsIHNlPVQsIGFscGhhPTAuNSkgKwogICAgZ2VvbV9wb2ludChzdGF0PSJpZGVudGl0eSIsIHBvc2l0aW9uPSJkb2RnZSIsIGFscGhhPTEsIHNpemU9MykgKwogICAgeWxhYihtZXRyaWMpICsKICAgIHhsYWIoIkxvZyBNaWxsaW9uIFRyYWluaW5nIFRva2VucyIpICsKICAgIGdndGl0bGUoIlRyYWluaW5nIFNpemUgdnMuIFByZWRpY3RpdmUgUG93ZXIiKSArCiAgICBmYWNldF9ncmlkKGNvcnB1c35tb2RlbCwgc2NhbGVzPSJmcmVlIikgKwogICAgI3NjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBjKCIjQTQyRUYxIiwgIiMzODk0QzgiKSkgKwogICAgdGhlbWUoYXhpcy50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIHN0cmlwLnRleHQueCA9IGVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGxlZ2VuZC50ZXh0PWVsZW1lbnRfdGV4dChzaXplPTE0KSwKICAgICAgICAgIGF4aXMudGl0bGU9ZWxlbWVudF90ZXh0KHNpemU9MTgpLAogICAgICAgICAgbGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIpCiNnZ3NhdmUoIi4vY29nc2NpX2ltYWdlcy90cmFpbmluZ19sb2dsaWsucG5nIixoZWlnaHQ9NSx3aWR0aD02KQpgYGAKCgpgYGB7ciBPbiBTRyBzY29yZX0KbW9kZWxfZGVsdGFzICU+JQogIG11dGF0ZSh0cmFpbl9zaXplID0gbG9nKHRyYWluX3NpemUpKSAlPiUKICBnZ3Bsb3QoYWVzKHg9dHJhaW5fc2l6ZSwgeT1zZ19zY29yZSwgY29sb3I9bW9kZWwpKSArCiAgICBnZW9tX3Ntb290aChtZXRob2Q9ImxtIiwgc2U9VCwgYWxwaGE9MC41KSArCiAgICBnZW9tX3BvaW50KHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249ImRvZGdlIiwgYWxwaGE9MSwgc2l6ZT0zKSArCiAgICB5bGFiKCJTRyBTQ29yZSIpICsKICAgIHhsYWIoIkxvZyBNaWxsaW9uIFRyYWluaW5nIFRva2VucyIpICsKICAgIGdndGl0bGUoIlRyYWluaW5nIFNpemUgdnMuIFN5bnRhY3RpYyBHZW5lcmFsaXphdGlvbiIpICsKICAgICNzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gYygiI0E0MkVGMSIsICIjMzg5NEM4IikpICsKICAgIGZhY2V0X2dyaWQofm1vZGVsLCBzY2FsZXM9ImZyZWUiKSArCiAgICB0aGVtZShheGlzLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgbGVnZW5kLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xOCksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIikKI2dnc2F2ZSgiLi9jb2dzY2lfaW1hZ2VzL3RyYWluaW5nX3NnLnBuZyIsaGVpZ2h0PTUsd2lkdGg9NikKYGBgCgojIyBTbWl0aCAmIExldnkgcmVwcm9kdWN0aW9uCgpgYGB7cn0KICAKYWxsX2RhdGEgJT4lCiAgZmlsdGVyKHN1cnByaXNhbCA8IDE1LCBzdXJwcmlzYWwgPiAwKSAlPiUKICBnZ3Bsb3QoYWVzKHg9c3VycHJpc2FsLCB5PXBzeWNob21ldHJpYywgY29sb3I9dHJhaW5pbmcpKSArCiAgICBzdGF0X3Ntb290aChzZT1ULCBhbHBoYT0wLjUpICsKICAgICNnZW9tX2Vycm9yYmFyKGNvbG9yPSJibGFjayIsIHdpZHRoPS4yLCBwb3NpdGlvbj1wb3NpdGlvbl9kb2RnZSh3aWR0aD0uOSksIGFscGhhPTAuMykgKwogICAgI2dlb21fcG9pbnQoc3RhdD0iaWRlbnRpdHkiLCBwb3NpdGlvbj0iZG9kZ2UiLCBhbHBoYT0xLCBzaXplPTMpICsKICAgIHlsYWIoIlByb2Nlc3NpbmcgVGltZSAobXMpIikgKwogICAgeGxhYigiU3VycHJpc2FsIChiaXRzKSIpICsKICAgIGdndGl0bGUoIlN1cnByaXNhbCB2cy4gUmVhZGluZyBUaW1lIC8gR2F6ZSBEdXJhdGlvbiIpICsKICAgIGZhY2V0X2dyaWQoY29ycHVzfm1vZGVsLCBzY2FsZXMgPSAiZnJlZSIpICsKICAgIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXMgPSBjKCJibGxpcC1sZyI9IiM0NDAxNTRGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1tZCI9IiMzOTU2OENGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1zbSI9IiMxRjk2OEJGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cyI9IiM3M0QwNTVGRiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC1sZy1ncHRicGUiPSIjODg4ODg4IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImJsbGlwLW1kLWdwdGJwZSI9IiNBQUFBQUEiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYmxsaXAtc20tZ3B0YnBlIj0iI0NDQ0NDQyIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJibGxpcC14cy1ncHRicGUiPSIjQ0NDQ0NDIikpICsKICAgIHRoZW1lKGF4aXMudGV4dD1lbGVtZW50X3RleHQoc2l6ZT0xNCksCiAgICAgICAgICBheGlzLnRleHQueSA9IGVsZW1lbnRfdGV4dChzaXplID0gMTApLAogICAgICAgICAgc3RyaXAudGV4dC54ID0gZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgbGVnZW5kLnRleHQ9ZWxlbWVudF90ZXh0KHNpemU9MTQpLAogICAgICAgICAgYXhpcy50aXRsZT1lbGVtZW50X3RleHQoc2l6ZT0xOCksCiAgICAgICAgICBsZWdlbmQucG9zaXRpb24gPSAicmlnaHQiKQpnZ3NhdmUoIi4uL2ltYWdlcy9jdW55MjAyMC9zdXJwX2NvcnIucG5nIixoZWlnaHQ9NC41LHdpZHRoPTEyKQpgYGAK